/*
 * Copyright 2011 David Crosson
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *   http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
 
package fr.janalyse.series

import java.io._
import scala.io._

import java.text.NumberFormat
import java.text.SimpleDateFormat
import java.util.Date
import java.util.Locale



private abstract sealed class PatternAndIndex {val index:Int; val pattern:String}

private case class DatePatternAndIndex(index:Int, pattern:String) extends PatternAndIndex
private case class TimePatternAndIndex(index:Int, pattern:String) extends PatternAndIndex
private case class DateTimePatternAndIndex(index:Int, pattern:String) extends PatternAndIndex

private case class CSVFormatInternals(
  separator:Char='\t',
  datePatterns:List[PatternAndIndex],
  valuesIndexes:List[Int],
  numberFormat:NumberFormat,
  pivotsIndexes:List[Int]) {

  def getDate(cells:Array[String]) : Date = {
    datePatterns match {
      case DatePatternAndIndex(i,pat)::Nil =>
          new SimpleDateFormat(pat).parse(cells(i))
      case DateTimePatternAndIndex(i,pat)::Nil =>
           new SimpleDateFormat(pat).parse(cells(i))
      case DatePatternAndIndex(id,pd)::TimePatternAndIndex(it,pt)::Nil =>
           new SimpleDateFormat(pd+" "+pt).parse(cells(id)+" "+cells(it))
      case TimePatternAndIndex(it,pt)::DatePatternAndIndex(id,pd)::Nil =>
           new SimpleDateFormat(pd+" "+pt).parse(cells(id)+" "+cells(it))
      case TimePatternAndIndex(it,pat)::Nil =>
           new SimpleDateFormat(pat).parse(cells(it))
      case _ =>
        throw new RuntimeException("datetime not recognized : '%s'".format(this))
    }
  }
 }


class CSVFormat(
  val separator:String,
  val lineSeparator:String,
  val numberFormat:NumberFormat,
  val dateTimeFormat:Option[SimpleDateFormat]
)

object CSVFormat {
  def apply(
	  separator:String=";",
	  lineSeparator:String="\n",
	  locale:Locale = Locale.ENGLISH,
	  dateTimeFormatPattern:Option[String]=Some("yyyy/MM/dd HH:mm:ss.SSS")
  ) = {
	val nf = NumberFormat.getNumberInstance(locale)
	nf.setGroupingUsed(false)
    nf.setMaximumIntegerDigits(Int.MaxValue)
	nf.setMinimumFractionDigits(0)
    nf.setMaximumFractionDigits(3) //Int.MaxValue
	new CSVFormat(separator, lineSeparator, nf, dateTimeFormatPattern map {new SimpleDateFormat(_, locale)})
  }
}

 
 
/**
  * CSV extractor with automatic CSV Format guess
  * @author David Crosson
  */
class CSV2Series(source:Source) {
  // ---------------------------------------------------------------------------
  private def guessSeparator(samples:List[String]):Option[Char] = {
    val candidates="\t;,"
    val counters = candidates map { sep => (sep -> (samples.map {_ count {_ == sep} } toSet) ) }

    counters find {
      case (sep, counts) if (counts.size==1 && counts.head > 0) => true
      case _ => false
    } match {
      case None => None
      case Some((sep,_)) => Some(sep)
    }
  }
  // ---------------------------------------------------------------------------
  private def guessDatePatterns(samples:List[String], separator:Char):List[PatternAndIndex] = {
    if (samples.size<=1) throw new RuntimeException("Not enough data to guess CSV file format")
    val time1RE="""\d{2}:\d{2}(:\d{2})?(?:([., ])\d+)?"""r
    val date1RE="""\d{4}([-/_])\d{2}\1\d{2}"""r
    val date2RE="""\d{2}([-/_])\d{2}\1\d{4}"""r
    val date3RE="""\d{2}([-/_])\d{2}\1\d{2}"""r
    val datetime1RE="""\d{4}([-/_])\d{2}\1\d{2}([ :])\d{2}:\d{2}(:\d{2})?(?:([.,])\d+)?"""r
    val datetime2RE="""\d{2}([-/_])\d{2}\1\d{4}([ :])\d{2}:\d{2}(:\d{2})?(?:([.,])\d+)?"""r
    val datetime3RE="""\d{2}([-/_])\d{2}\1\d{2}([ :])\d{2}:\d{2}(:\d{2})?(?:([.,])\d+)?"""r
    val sample=samples.tail.head // 1ere ligne de donnees
    val cells=sample.split(separator.toString, -1).toList map {_.trim}
    def checkS(withS:String) = withS match { case null => "" case _ => ":ss"}
    val patterns = (cells zipWithIndex) flatMap {
      case (time1RE(withS,null), i)           => Some(TimePatternAndIndex(i,"HH:mm%s".format(checkS(withS))))
      case (time1RE(withS,ms),i)              => Some(TimePatternAndIndex(i,"HH:mm%s%sS".format(checkS(withS),ms)))
      case (date1RE(ds),i)                    => Some(DatePatternAndIndex(i,"yyyy%sMM%sdd".format(ds,ds)))
      case (date2RE(ds),i)                    => Some(DatePatternAndIndex(i,"dd%sMM%syyyy".format(ds,ds)))
      case (date3RE(ds),i)                    => Some(DatePatternAndIndex(i,"yy%sMM%sdd".format(ds,ds)))
      case (datetime1RE(ds,ts,withS,null),i)  => Some(DateTimePatternAndIndex(i,"yyyy%sMM%sdd%sHH:mm%s".format(ds,ds,ts,checkS(withS))))
      case (datetime1RE(ds,ts,withS,ms),i)    => Some(DateTimePatternAndIndex(i,"yyyy%sMM%sdd%sHH:mm%s%sS".format(ds,ds,ts,checkS(withS),ms)))
      case (datetime2RE(ds,ts,withS,null),i)  => Some(DateTimePatternAndIndex(i,"dd%sMM%syyyy%sHH:mm%s".format(ds,ds,ts,checkS(withS))))
      case (datetime2RE(ds,ts,withS,ms),i)    => Some(DateTimePatternAndIndex(i,"dd%sMM%syyyy%sHH:mm%s%sS".format(ds,ds,ts,checkS(withS),ms)))
      case (datetime3RE(ds,ts,withS,null),i)  => Some(DateTimePatternAndIndex(i,"dd%sMM%syy%sHH:mm%s".format(ds,ds,ts,checkS(withS))))
      case (datetime3RE(ds,ts,withS,ms),i)    => Some(DateTimePatternAndIndex(i,"dd%sMM%syy%sHH:mm%s%sS".format(ds,ds,ts,checkS(withS),ms)))
      case _ => None
    }
    patterns
  }
  private def guessValues(samples:List[String], separator:Char, indexesRemaining:List[Int]) : Tuple2[List[Int], NumberFormat] = {
    val N1="""\d+"""r
    val N2="""\d+(?:[eE]\d+)?"""r
    val N3="""\d+([,.])\d*(?:[eE]\d+)?"""r
    val N4="""([,.])\d+(?:[eE]\d+)?"""r
    var numberIndexes=Set[Int]()
    var decSep=Set.empty[String]

    for( sample:String <- samples;
        (cell, index) <- sample.split(separator.toString, -1).zipWithIndex if (indexesRemaining contains index)
    ) cell.trim match {
      case N1()   => numberIndexes+=index
      case N2()   => numberIndexes+=index
      case N3(ds) => numberIndexes+=index ; decSep+=ds
      case N4(ds) => numberIndexes+=index ; decSep+=ds
      case ""     => numberIndexes+=index   // Empty means no value
      case _ =>
    }

    if (decSep.size>1) throw new RuntimeException("Couln't compute used decimal separator")

    val nf = NumberFormat.getNumberInstance(if (decSep contains ".") Locale.US else Locale.FRANCE)

    (numberIndexes.toList, nf)
  }
  // ---------------------------------------------------------------------------
  private def guessFormat(samples:List[String]): CSVFormatInternals = {
    guessSeparator(samples) match {
      case None => throw new Exception("Couln't guess csv cell separator")
      case Some(separator) =>
        val cellCount = samples.head.split(separator.toString, -1).size
        val datePatterns = guessDatePatterns(samples, separator)
        val dateIndexes = datePatterns map {
          case DatePatternAndIndex(i,_)=>i
          case TimePatternAndIndex(i,_)=>i
          case DateTimePatternAndIndex(i,_)=>i
        }
        var remainingIndexes = (0 until cellCount) filterNot {dateIndexes contains _} toList
        val (valuesIndexes, numberFormat) = guessValues(samples,  separator, remainingIndexes)
        val pivotsIndexes = remainingIndexes filterNot {valuesIndexes contains _} toList // Si >0 alors presence de pivots

        CSVFormatInternals(separator, datePatterns, valuesIndexes, numberFormat, pivotsIndexes)
    }
  }
  // ---------------------------------------------------------------------------
  def extract : Map[String, Series[CalcCell]] = {
    val samples = source.reset.getLines.slice(0,100).toList filter {_.size > 0}
    val format = guessFormat(samples)
    val headers = samples.head.split(format.separator.toString, -1).toArray map {_.trim}
    var seriesMap = Map.empty[String, Series[CalcCell]]
    def makeSeriesName(cells:Array[String], header:String) = {
      val basename = format.pivotsIndexes map {cells(_)} mkString("-")
      if (basename.size>0) basename+"-"+header else header
    }

    for (line <- source.reset.getLines.drop(1) if line.trim.size>0) {
      val cells = line.split(format.separator.toString, -1) map {_ trim}  // -1 to say split all do not trim anything!!
      val date = format.getDate(cells)
      for(valueIndex <- format.valuesIndexes if valueIndex < cells.size ; header = headers(valueIndex); cell = cells(valueIndex)) {
        val seriesName = makeSeriesName(cells, header)
        var series = seriesMap.getOrElse(seriesName, Series[CalcCell](seriesName, new TimeModel(1L)))
        cell.trim match {
          case "" => // Empty Cell, ignoring
          case _ =>
            val value = format.numberFormat.parse(cell).doubleValue
            series = series << date.getTime -> value
            seriesMap += seriesName -> series
        }
      }
    }
    seriesMap
  }
}



/**
  * CSV2Series companion object
  * @author David Crosson
  */

case class FileContainer(file:File)
object FileContainer {
	implicit def file2FileContainer(f:File)     = FileContainer(f)
	implicit def string2FileContainer(s:String) = FileContainer(new File(s))
}


object CSV2Series {
  import java.net.URL
  def fromFile(fc:FileContainer) = new CSV2Series(Source.fromFile(fc.file)).extract
  def fromString(data:String)    = new CSV2Series(Source.fromString(data)).extract
  def fromURL(url:URL)           = new CSV2Series(Source.fromURL(url)).extract
  def fromURL(s:String)          = new CSV2Series(Source.fromURL(new URL(s))).extract
  
  implicit val defaultCSVFormat:CSVFormat = CSVFormat()

  def toString[C<:Cell](seriesMap:Map[String, Series[C]]):String = toString(seriesMap.values, defaultCSVFormat) 
  def toString[C<:Cell](seriesMap:Map[String, Series[C]], format:CSVFormat):String = toString(seriesMap.values, format) 
  def toString[C<:Cell](series:Series[C]):String = toString(series::Nil, defaultCSVFormat)
  def toString[C<:Cell](series:Series[C], format:CSVFormat):String = toString(series::Nil, format)
  def toString[C<:Cell](seriesList:Iterable[Series[C]]):String = toString(seriesList, defaultCSVFormat)
  def toString[C<:Cell](seriesList:Iterable[Series[C]], format:CSVFormat) = {
    val output = new StringWriter()
    toWriter(seriesList, output, format)
    output.toString
  }
  
  def toFile[C<:Cell](seriesMap:Map[String,Series[C]], dest:FileContainer) = toWriter(seriesMap.values, new FileWriter(dest.file))
  def toFile[C<:Cell](seriesList:Iterable[Series[C]], dest:FileContainer) = toWriter(seriesList, new FileWriter(dest.file))
  def toFile[C<:Cell](series:Series[C], dest:FileContainer) = toWriter(series::Nil, new FileWriter(dest.file))

  def toFile[C<:Cell](seriesMap:Map[String,Series[C]], dest:FileContainer, format:CSVFormat) = toWriter(seriesMap.values, new FileWriter(dest.file), format)
  def toFile[C<:Cell](seriesList:Iterable[Series[C]], dest:FileContainer, format:CSVFormat) = toWriter(seriesList, new FileWriter(dest.file), format)
  def toFile[C<:Cell](series:Series[C], dest:FileContainer, format:CSVFormat) = toWriter(series::Nil, new FileWriter(dest.file), format)

  def toWriter[C<:Cell](seriesList:Iterable[Series[C]], output:Writer, format:CSVFormat=defaultCSVFormat) = {
    val pout = new PrintWriter(output)
    val sep  = format.separator
    val nl   = format.lineSeparator
    val sdf  = format.dateTimeFormat
    val nf   = format.numberFormat

    var current = seriesList map { _.toList}
    val headers = seriesList map { _.name}
    pout.print("DateTime")
    pout.print(headers.mkString(sep,sep,""))
    pout.print(nl)
    var moretodo=true
    do {
      val heads   = current map {_.headOption}
      val cells   = heads filter {_.isDefined} map {_.get}
      
      if (cells.isEmpty) moretodo = false
      else {
        var emptyline=true
        val minTime = (cells map {_.time}).min
        val headsAsStrings = heads map {head =>
          if (head.isDefined) {
            val value = head.get.value
            if (head.get.time == minTime && !value.isNaN && !value.isInfinite) {
              emptyline=false
              nf.format(value)
            } else {
              ""
            }
          } else {
            ""
          }
        }
        current = current map {s => 
            s.headOption match {
               case Some(head) if (head.time==minTime) => s.tail
               case _ => s
            }
        }
        if (!emptyline) {
          pout.print(sdf map {_.format(minTime)} getOrElse minTime)
          pout.print(headsAsStrings.mkString(sep,sep,""))
          pout.print(nl)
        }
      }
    } while(moretodo)
    pout.close
  }
}



// ==============================================================================================================
